#include "tokenizer.hpp"
#include <sentencepiece_processor.h>
#include <stdexcept>

class TokenizerImpl: public Tokenizer {
private:
    sentencepiece::SentencePieceProcessor sp;
public:
    TokenizerImpl(const std::string &tok_path) {
        if (!sp.Load(tok_path).ok()) {
            throw std::runtime_error("Failed to load tokenizer model from " + tok_path);
        }
    }
    std::vector<int> encode(const std::string &text) override {
        return sp.EncodeAsIds(text);
    }
    std::string decode(const std::vector<int> &tokens) override {
        return sp.DecodeIds(tokens);
    }
};

std::unique_ptr<Tokenizer> create_tokenizer(const std::string &tok_path) {
    return std::make_unique<TokenizerImpl>(tok_path);
}
